# -*- coding: UTF-8 -*-
# Author: Damon(wuud1)
# CreateDate: 2018-08-07
# Message: 代码重构, 每次get请求的MD5值增加local_pid作为唯一区分,从而避免掉一部分bug,增添最新版断点续爬功能,main函数集中控制
# 替换了最新版parse_url函数, 没有修改原代码的数据提取逻辑, 请求头没有精简,
import re
import requests
import redis
import os
import sys
import hashlib
import pymysql
import json
from lxml import etree
import time
from collections import OrderedDict

# 解决python2编码问题
reload(sys)
sys.setdefaultencoding('utf-8')


class MahleForPid(object):

    def __init__(self, **kwargs):
        self.db_config = kwargs.get("db_config")  # 数据库配置
        self.select_table = kwargs.get("select_table")  # 查询表名
        self.insert_table = kwargs.get("insert_table")  # 插入表名
        self.partBrand = kwargs.get("partBrand")  # 网站品牌
        self.redis_set = '{}_set'.format(kwargs.get('partBrand'))  # redis数据库集合名,用来存储查询过的pid
        self.conn1 = None  # 数据库链接和游标
        self.conn2 = None
        self.curs1 = None
        self.curs2 = None
        self.base_path = '/Users/duanmengdi/Desktop/workspace/{}_html/'.format(kwargs.get("partBrand"))  # 存储html文件的路径
        self.location = ""  # 重定向链接
        self.total_num = 0  # 全局计数变量
        self.is_sleep = kwargs.get("is_sleep", True)  # 是否控制速度的开关变量
        self.r = redis.Redis(host='localhost', port=6379, db=0).pipeline()  # redis链接,用于断点续爬

        # 请求所需的参数,可精简,未精简,懒得精简
        self.headers = {
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
            "Accept-Encoding": "gzip, deflate",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Cache-Control": "max-age=0",
            "Connection": "keep-alive",
            "Content-Type": "application/x-www-form-urlencoded",
            "Host": "catalog.mahle-aftermarket.cn",
            "Origin": "http://catalog.mahle-aftermarket.cn",
            "Referer": "http://catalog.mahle-aftermarket.cn/cn/modules/oeFilter/index.xhtml",
            "Upgrade-Insecure-Requests": "1",
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
        }
        self.last_headers = {
            "Accept": "*/*",
            "Accept-Encoding": "gzip, deflate",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Cache-Control": "no-cache",
            "Connection": "keep-alive",
            "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
            "Host": "catalog.mahle-aftermarket.cn",
            "Origin": "http://catalog.mahle-aftermarket.cn",
            "Pragma": "no-cache",
            "Referer": "",
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
            "Proxy-Connection": "keep-alive",
        }

        self.cookies = {
            "__utmz": "94631182.1531893131.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)",
            "__utmc": "94631182",
            "__utma": "94631182.70392564.1531893131.1533613807.1533621215.9",
            "__utmb": "94631182.0.10.1533621215",
            "JSESSIONID": "296EFB6B1BC5095E401DE7519DE71DEC",
            # "": "",
        }

        self.search_post_data = OrderedDict()  # 搜索发送的data
        self.search_post_data['autoScroll'] = '',
        self.search_post_data['statistic_ftsInputFeld'] = '',
        self.search_post_data['validationFtsModalBoxOpenedState'] = '',
        self.search_post_data['waitModalBoxOpenedState'] = '',
        self.search_post_data['statistic_bezeichnung'] = '',
        self.search_post_data['j_id13_SUBMIT'] = '1',
        self.search_post_data['javax.faces.ViewState'] = '',
        self.search_post_data['j_id13:_idcl'] = 'searchButton',

        self.last_post_data = OrderedDict()
        self.last_post_data['AJAXREQUEST'] = ''
        self.last_post_data['autoScroll'] = ''
        self.last_post_data['statistic_ftsInputFeld'] = ''
        self.last_post_data['validationFtsModalBoxOpenedState'] = ''
        self.last_post_data['waitModalBoxOpenedState'] = ''
        self.last_post_data['j_id13_SUBMIT'] = '1'
        self.last_post_data['javax.faces.ViewState'] = ''
        self.last_post_data['hersteller'] = ''
        self.last_post_data['filterId'] = ''
        self.last_post_data['show'] = 'oeFilter'
        self.last_post_data['ajaxSingle'] = 'hersteller'

        self.index_url = "http://catalog.mahle-aftermarket.cn/cn/modules/oeFilter/index.xhtml"
        self.search_url = "http://catalog.mahle-aftermarket.cn/cn/modules/oeFilter/index.xhtml"
        self.base_url = 'http://catalog.mahle-aftermarket.cn'
        self.last_post_url = 'http://catalog.mahle-aftermarket.cn/cn/modules/mahleFilter/detail.xhtml'
        self.javax_value = ""


    def getmd5(self, str):
        '''获取md5加密电子指纹'''
        md5 = hashlib.md5()
        md5.update(str)
        return md5.hexdigest()

    def get_conn(self, dbconfig_dict):
        '''获取数据库链接'''
        conn = pymysql.connect(**dbconfig_dict)
        return conn

    def get_cursor(self, conn, type='stream'):
        '''获取数据库游标'''
        if type == 'stream':
            return conn.cursor(pymysql.cursors.SSCursor)  # 返回流式游标,查询大量数据时不占用内存(返回数据形式是元组)
        elif type == 'dict':
            return conn.cursor(pymysql.cursors.DictCursor)  # 返回字典形式游标,查询出的数据以字典形式返回
        else:
            raise Exception("cursor type error")

    def parse_url(self, url, method, post_data=None, headers=None, tofile=True, **kwargs):
        requests.packages.urllib3.disable_warnings()  # 因为把ssl安全验证关了,所以不让代码报不安全的https请求警告
        pid = kwargs.get("local_pid", "")
        charset = kwargs.get("charset", "utf8")  # 暂时没有用到
        allow_redirects = kwargs.get("allow_redirects", False)  # 是否允许重定向, 默认不重定向,获取原始响应
        if method == 'GET':
            data = "GET" + url + pid  # 拼接data
            md5 = self.getmd5(data)  # 获取此次请求电子指纹
            file_path = self.base_path + md5 + ".html"
            if os.path.exists(file_path):  # 判断文件是否存在,如果存在就从本地读取
                print "GET for local"
                with open(file_path, 'r') as f:
                    res = f.read()
                response = res
            else:  # 如果路径不存在, 从web端请求
                print "GET for web"
                try:
                    response = requests.get(url=url, verify=False, headers=headers, cookies=self.cookies, timeout=30,
                                            allow_redirects=allow_redirects)
                except Exception as e:
                    response = requests.get(url=url, verify=False, headers=headers, cookies=self.cookies, timeout=30,
                                            allow_redirects=allow_redirects)

                # 每次请求都获取cookies, 并更新self.cookies(如果获取到cookies的话)
                cookies = response.cookies.get_dict()
                for k in cookies:
                    self.cookies[k] = cookies[k]

                # 获取location,
                location = response.headers.get("Location", None)
                if location:
                    print "响应头中存在location"
                    self.location = location
                    # return response.content.decode('utf8')

                if response:
                    if tofile:
                        with open(file_path, 'w') as f:  # 保存此次请求的html
                            f.write(response.content.decode('utf8'))
                    response = response.content.decode("utf8")
                else:
                    response = None

        elif method == "POST":
            data = "POST" + url + json.dumps(post_data)
            md5 = self.getmd5(data)
            file_path = self.base_path + md5 + ".html"
            if os.path.exists(file_path):  # 判断文件是否存在,如果存在就从本地读取
                print "POST for local"
                with open(file_path, 'r') as f:
                    res = f.read()
                response = res
            else:
                print "POST for web"
                try:
                    response = requests.post(url=url, data=post_data, verify=False, headers=headers,
                                             cookies=self.cookies, timeout=30,
                                             allow_redirects=allow_redirects)
                except Exception as e:
                    response = requests.post(url=url, data=post_data, verify=False, headers=headers,
                                             cookies=self.cookies, timeout=30,
                                             allow_redirects=allow_redirects)

                # 获取cookies, 并更新self.cookies
                cookies = response.cookies.get_dict()
                for k in cookies:
                    self.cookies[k] = cookies[k]

                # 获取location,
                location = response.headers.get("Location", None)
                if location:
                    print "响应头中存在location"
                    self.location = location
                    # response, file_path = self.parse_url(url=location, method="GET", headers=headers, local_pid=pid)
                    # print "已重新请求location地址"
                    # return response, file_path
                    # return response.content.decode('utf8')

                if response:
                    if tofile:
                        with open(file_path, 'w') as f:  # 保存此次请求的html
                            f.write(response.content.decode('utf8'))
                    response = response.content.decode("utf8")

                else:
                    response = None

        return response, file_path

    def save2db(self, data_dict, curs):
        # 抽取数据
        pid = data_dict['pid'].decode('utf8')
        brand_name = data_dict['brand_name'].decode('utf8')
        part_brand = data_dict['part_brand'].decode('utf8')
        part_num = data_dict['part_num'].decode('utf8')
        part_name = data_dict['part_name'].decode('utf8')
        brand_name1 = data_dict['brand_name1'].decode('utf8')
        pid1 = data_dict['pid1'].decode('utf8')
        status = data_dict['status'].decode('utf8')

        # 拼接sql语句
        sql = '''INSERT INTO `{}` (`pid`, `brand_name`, `part_brand`, `part_num`, `part_name`, `brand_name1`, `pid1`, `status`) VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s')'''.format(self.insert_table) % (
        pid, brand_name, part_brand, part_num, part_name, brand_name1, pid1, status)
        # print sql
        try:
            curs.execute(sql)
        except Exception as e:
            print e
            print sql
            exit()
        self.total_num += 1  # 数据总量计数

    def r_reset(self):
        '''删除redis集合,起到重置断点续爬的功能'''
        self.r.delete(self.redis_set)
        self.r.execute()

    def run(self):
        # 获取一系列数据库链接,游标
        self.conn1 = self.get_conn(dbconfig_dict=self.db_config)
        self.conn2 = self.get_conn(dbconfig_dict=self.db_config)
        self.curs1 = self.get_cursor(conn=self.conn1)
        self.curs2 = self.get_cursor(conn=self.conn2)
        select_sql = '''select pid,brand from {}'''.format(self.select_table)

        self.curs1.execute(select_sql)
        for data in self.curs1:
            local_pid = data[0]  # 当前条数据的pid
            local_pid = 'LR011279'  # 测试用数据,有数据

            # 断点续爬代码块
            self.r.sismember(self.redis_set, local_pid)
            if self.r.execute()[0]:  # 在set里返回true, 否则返回false
                continue
            self.r.sadd(self.redis_set, local_pid)  # 把pid添加到set中

            # 控制速度
            if self.is_sleep:
                time.sleep(1)

            brand_name = data[1]

            # 开始访问入口(首页)
            index_response, index_file_path = self.parse_url(url=self.index_url, method="GET", headers=self.headers, local_pid=local_pid)
            print index_file_path
            if index_response:
                index_html = etree.HTML(index_response)
            else:
                print local_pid, index_file_path
                raise Exception("没有获取到首页响应")

            self.javax_value = index_html.xpath("//input[@id='javax.faces.ViewState']/@value")
            self.javax_value = self.javax_value[0] if len(self.javax_value)>0 else ""
            # 更新headers
            self.search_post_data['javax.faces.ViewState'] = self.javax_value

            # 拼装search_post_data
            self.search_post_data['statistic_bezeichnung'] = local_pid
            search_result_response, search_result_file_path = self.parse_url(url=self.search_url, method="POST", post_data=self.search_post_data, headers=self.headers)

            if search_result_response:
                search_result_html = etree.HTML(search_result_response)
            else:
                print local_pid, search_result_file_path
                raise Exception("没有获取到搜索结果页响应")

            detail_url_list = search_result_html.xpath("//a[@class='detailLink']/@href")  # 获取详情页面url
            if len(detail_url_list) <= 0:
                print "没有找到详情url, 跳过", local_pid, search_result_file_path

            for detail_url in detail_url_list:
                detail_url = self.base_url + detail_url
                print detail_url

                detail_reaponse, detail_file_path = self.parse_url(url=detail_url, method="GET", headers=self.headers, local_pid=local_pid)

                if detail_reaponse:
                    detail_html = etree.HTML(detail_reaponse)
                else:
                    print local_pid, detail_file_path
                    raise Exception("没有获取到详情页响应")

                # 获取零件基础数据
                part_data = detail_html.xpath("//td[@class='data'][1]/input/@value")
                part_num = part_data[0].replace(' ', '')
                part_name = part_data[1].replace(' ', '')

                oefilter_url = detail_url + "&show=oeFilter"  # 拼接进入OE型号标签界面的url

                oe_response, oe_file_path = self.parse_url(url=oefilter_url, method="GET", headers=self.headers, local_pid=local_pid)

                if oe_response:
                    oe_html = etree.HTML(oe_response)
                else:
                    print local_pid, oe_file_path
                    raise Exception("没有获取到oe对照页响应")

                # 先获取显示出来的数据
                oe_tr_list = oe_html.xpath("//tbody[contains(@id,':tbody_element')]/tr")
                for tr in oe_tr_list:
                    brand_name1 = tr.xpath("./td[1]/text()")
                    pid1 = tr.xpath("./td[2]/text()")
                    brand_name1 = brand_name1[0] if len(brand_name1) > 0 else ""
                    pid1 = pid1[0] if len(pid1) > 0 else ""
                    if pid1:
                        pid1 = pid1.replace(" ", '')

                    last_data = dict(
                        pid=local_pid,
                        brand_name=brand_name,
                        part_brand=self.partBrand,  # mahle的网站,当然都是mahle
                        part_num=part_num,
                        part_name=part_name,
                        brand_name1=brand_name1,
                        pid1=pid1,
                        status=''
                    )
                    self.save2db(last_data, self.curs2)
                    print last_data

                    if self.total_num % 100 == 0:
                        self.conn2.commit()
                        self.r.execute()
                        print "提交insert缓存, 当前数量: {}".format(self.total_num)

                # 获取一些必须得参数
                select = oe_html.xpath("//select[@id='hersteller']")
                select = select[0] if len(select) > 0 else None
                if select is None:
                    print "没有找到select下拉菜单, 跳过", local_pid
                    continue
                onchange = select.xpath("./@onchange")
                onchange = onchange[0] if len(onchange) > 0 else None
                if onchange is None:
                    print "没有找到包含的必要参数, 跳过", local_pid
                    continue
                print "最后一次post请求所必要的参数", onchange

                # 获取一些必要的参数(都是请求里需要的参数)
                AJAXREQUEST = re.findall("'containerId':'(.*?)'", onchange)
                AJAXREQUEST = AJAXREQUEST[0] if len(AJAXREQUEST) > 0 else ""
                filterId = re.findall("'filterId':'(.*?)'", onchange)
                filterId = filterId[0] if len(filterId) > 0 else ""
                jid = re.findall("j_id.*?(j_id.*?)'", onchange)
                jid = jid[0] if len(jid) > 0 else ""

                self.javax_value = oe_html.xpath("//input[@id='javax.faces.ViewState']/@value")
                self.javax_value = self.javax_value[0] if len(self.javax_value) > 0 else ""

                option_list = select.xpath("./option")  # 获取option列表(车型下拉菜单)
                option_list.pop(0)  # 去除第一个已经拿过数据的车型
                for option in option_list:
                    brand_name1 = option.xpath("./text()")
                    brand_name1 = brand_name1[0] if len(brand_name1) > 0 else None
                    hersteller = option.xpath("./@value")
                    hersteller = hersteller[0] if len(hersteller) > 0 else None
                    if brand_name1 is None or hersteller is None:
                        print "没有找到brand_name1或者对应的value", local_pid
                        continue

                    # # 拼装最终的post请求
                    self.last_post_data['AJAXREQUEST'] = AJAXREQUEST
                    self.last_post_data['filterId'] = filterId
                    self.last_post_data[jid] = jid
                    self.last_post_data['hersteller'] = hersteller
                    self.last_post_data['javax.faces.ViewState'] = self.javax_value

                    # 更新last_headers里的referer
                    self.last_headers['Referer'] = oefilter_url

                    last_response, last_file_path = self.parse_url(url=self.last_post_url, method="POST", post_data=self.last_post_data,
                                                   headers=self.last_headers, local_pid=local_pid)

                    if last_response:
                        last_html = etree.HTML(last_response)
                    else:
                        print local_pid, brand_name1, last_file_path
                        raise Exception("没有获取到最终数据响应")

                    # 获取剩下的车型最终数据
                    tr_list = last_html.xpath("//tbody[contains(@id,':tbody_element')]/tr")
                    for tr in tr_list:
                        brand_name1 = tr.xpath("./td[1]/text()")
                        pid1 = tr.xpath("./td[2]/text()")
                        brand_name1 = brand_name1[0] if len(brand_name1) > 0 else ""
                        pid1 = pid1[0] if len(pid1) > 0 else ""
                        if pid1:
                            pid1 = pid1.replace(" ", '')

                        last_data = dict(
                            pid=local_pid,
                            brand_name=brand_name,
                            part_brand='mahle',  # mahle的网站,当然都是mahle
                            part_num=part_num.replace(' ', ''),
                            part_name=part_name,
                            brand_name1=brand_name1.replace(' ', ''),
                            pid1=pid1,
                            status=''
                        )
                        print last_data
                        self.save2db(last_data, self.curs2)

                        if self.total_num % 100 == 0:
                            self.conn2.commit()
                            self.r.execute()
                            print "提交insert缓存, 当前数量: {}".format(self.total_num)
            # exit()
        self.conn2.commit()
        self.r.execute()
        print "最终提交, 总数据量: {}".format(self.total_num)

        self.curs1.close()
        self.curs2.close()
        self.conn1.close()
        self.conn2.close()


if __name__ == '__main__':
    db_config = dict(
        host="127.0.0.1",
        port=3306,
        user="root",
        passwd="123456",
        db="my_data",
        charset="utf8"
    )
    select_table = '0716_pid'  # 查询表
    insert_table = 'mahle_all'  # 插入表
    partBrand = 'mahle'  # 网站品牌
    is_sleep = False  # 控制速度开关

    tools = MahleForPid(db_config=db_config, select_table=select_table, insert_table=insert_table, partBrand=partBrand,
                         is_sleep=is_sleep)
    # tools.r_reset()  # 重置断点续爬, 重置启动后请立即屏蔽,防止下次不小心又重置
    tools.run()  # 启动